#include<complex.h>
#include<fftw3.h>
#include<stdlib.h>
#include<unistd.h>
#include<assert.h>
#include<limits.h>
#include<math.h>
#include<string.h>
#include<stdio.h>
#include<time.h>
#include<rav.h>

/* allocation */

rav_hash_t * 
rav_init(int random_seed, int max_elts, int vect_entries)
{
   /* check limits */
   if (max_elts<=0)
      max_elts = DEFAULT_MAX_RAV_ELTS;
   if (vect_entries<=0)
      vect_entries = DEFAULT_RAV_ELT_VECT_ENTRIES;
   if (random_seed<=0)
      random_seed = (int)time(0);
   /* allocate space */
   rav_hash_t *h = calloc(1, sizeof(rav_hash_t));
   if (!h)
      return 0;
   h->hash_ary = calloc(RAV_HASH_ENTRIES, sizeof(rav_hash_elt_t));
   if (!h->hash_ary)
      return 0;
   h->env_ary = calloc(max_elts, sizeof(double)*vect_entries);
   if (!h->env_ary)
      return 0;
   h->ctx_ary = calloc(max_elts, sizeof(double)*vect_entries);
   if (!h->ctx_ary)
      return 0;
   h->name_ary = calloc(max_elts, sizeof(char*));
   if (!h->name_ary)
      return 0;
   h->low_freq_size = ((max_elts/(sizeof(uint64_t)*8))+1);
   h->low_freq = calloc(h->low_freq_size, sizeof(uint64_t));
   if (!h->low_freq)
      return 0;
   h->cur_idx = 0;
   h->total_name_size = 0;
   /* init random number generator */
   srandom((unsigned)random_seed);
   h->orig_random_seed = h->cur_random_seed = random_seed;
   h->dim = vect_entries;
   h->max_elts = max_elts;
   h->inv_sqrt_max_elts = 1./sqrt(max_elts);
   return h;
}

void
rav_free_elts(rav_hash_t *h)
{
   rav_hash_elt_t *e;
   rav_hash_elt_t *prev_e;
   int i;
   for (i=0; i<RAV_HASH_ENTRIES; i++) {
      for (e=&h->hash_ary[i]; e->next!=0;) {
         prev_e = e;
         e=e->next;
         if (prev_e!=&h->hash_ary[i])
            free(prev_e);
      }
      if (e!=&h->hash_ary[i])
         free(e);
   }
}

void
rav_free_names(rav_hash_t *h)
{
   int i;
   for (i=0; i<h->cur_idx; i++) {
      if (h->name_ary[i]!=0)
         free(h->name_ary[i]);
   }
}

void
rav_destroy(rav_hash_t *h)
{
   rav_free_elts(h);
   rav_free_names(h);
   free(h->low_freq);
   free(h->name_ary);
   free(h->ctx_ary);
   free(h->env_ary);
   free(h->hash_ary);
   free(h);
}

/* helpers for freq code access */

int 
get_freq_code(rav_hash_t *h, int idx)
{
   return (h->low_freq[idx>>LOW_FREQ_SHIFT]&(1LL<<(idx&LOW_FREQ_MASK)))!=0;
}

void 
set_freq_code(rav_hash_t *h, int idx, int v)
{
   if (v)
      h->low_freq[idx>>LOW_FREQ_SHIFT] |= (1LL<<(idx&LOW_FREQ_MASK));
   else
      h->low_freq[idx>>LOW_FREQ_SHIFT] &= ~(1LL<<(idx&LOW_FREQ_MASK));
}

/* misc helpers */

double
frand(void)
{
   return ((double)random())/(double)INT_MAX;
}

/* http://www.bearcave.com/misl/misl_tech/wavelets/hurst/random.html */
double
gauss(const double sigma)
{
   double x, y, r2;

   do {
      /* choose x,y in uniform square (-1,-1) to (+1,+1) */
      x = -1 + 2 * frand();
      y = -1 + 2 * frand();
      /* see if it is in the unit circle */
      r2 = x * x + y * y;
   } while (r2 > 1.0 || r2 == 0);
   /* Box-Muller transform */
   return sigma * y * sqrt (-2.0 * log (r2) / r2);
}

/* http://www.cse.yorku.ca/~oz/hash->html */
unsigned int
calc_hash(const char *str)
{
   unsigned int hash = 5381;
   int c;

   while ((c = (unsigned char)*str++))
      hash = ((hash << 5) + hash) + c; /* hash * 33 + c */
   hash = (hash>>16)^(hash&0xffff);
   return hash;
}

/* adding/accessing */

void
rav_init_value(rav_hash_t *h, rav_index_t idx)
{
   double *e, *c;
   int i;
   for (e = &h->env_ary[idx*h->dim],
        c = &h->ctx_ary[idx*h->dim],
        i=0; i<h->dim; i++) {
      *e++ = (double)gauss(h->inv_sqrt_max_elts);
      *c++ = 0.;
   }
}

rav_index_t
rav_new(rav_hash_t *h, const char *s, int freq_code)
{
   if (freq_code==-1)
      freq_code = FREQ_OK;
   /* check for overflow */
   if (h->cur_idx>=(h->max_elts-1)) { // keep last as temp buffer
      fprintf(stderr, "rav_new: h->cur_idx>=h->max_elts\n");
      return -1;
   }
   /* go ahead and add it */
   rav_init_value(h, h->cur_idx);
   unsigned int hash = calc_hash(s);
   rav_hash_elt_t *e = &h->hash_ary[hash];
   if (e->ctx!=0) {
      /* first entry in use */
      while (e->next!=0) {
         e = e->next;
      }
      e->next = (rav_hash_elt_t*)malloc(sizeof(rav_hash_elt_t));
      if (e->next==0) {
         fprintf(stderr, "rav_new: hash elt malloc failed at cur_idx=%ld\n", h->cur_idx);
         return -1; /* malloc failed */
      }
      e = e->next;
   }
   /* e now points to entry where we can add */
   e->next = 0;
   e->ctx = &h->ctx_ary[h->cur_idx*h->dim];
   set_freq_code(h, h->cur_idx, freq_code);
   int s_len = strlen(s)+1;
   h->total_name_size += s_len;
   char *nm = h->name_ary[h->cur_idx] = (char *)malloc(s_len);
   if (nm==0) {
      e->ctx = 0;
      fprintf(stderr, "rav_new: string malloc failed at cur_idx=%ld\n", h->cur_idx);
      return -1;
   }
   (void)strcpy(nm, s);
   return h->cur_idx++;
}

rav_index_t
rav_lookup_(rav_hash_t *h, const char *s, int reason, rav_hash_elt_t **elt,
      int *elt_is_target)
{
   if (!*s)
      return -1;
   if (reason==-1)
      reason = LOOKUP_FEATURE;
   if (elt_is_target)
      *elt_is_target = 1;
   unsigned int hash = calc_hash(s);
   rav_hash_elt_t *e = &h->hash_ary[hash];
   rav_index_t idx;
   if (e->ctx!=0) { // not empty
      idx = (rav_index_t)((e->ctx-h->ctx_ary)/h->dim);
      if (!strcmp(h->name_ary[idx], s)) {
         if (elt)
            *elt = e;
         if (reason==LOOKUP_FEATURE)
            return idx;
         else if (get_freq_code(h, idx)==FREQ_OK)
            return idx;
         else
            return -1;
      }
      if (elt_is_target)
         *elt_is_target = 0;
      while (e->next!=0) {
         if (elt)
            *elt = e;
         e = e->next;
         idx = (rav_index_t)((e->ctx-h->ctx_ary)/h->dim);
         if (!strcmp(h->name_ary[idx], s)) {
            if (reason==LOOKUP_FEATURE)
               return idx;
            else if (get_freq_code(h, idx)==FREQ_OK)
               return idx;
            else
               return -1;
         }
      }
   }
   return -1;
}

rav_index_t
rav_lookup(rav_hash_t *h, const char *s, int reason)
{
   return rav_lookup_(h, s, reason, 0, 0);
}

char *
rav_get_name(rav_hash_t *h, rav_index_t idx)
{
   return h->name_ary[idx];
}

long int
rav_num_unique(rav_hash_t *h)
{
   return h->cur_idx;
}

long int
rav_get_tmp_idx(rav_hash_t *h)
{
   return h->cur_idx;
}

int
rav_delete(rav_hash_t *h, const char *s)
{
   // FIXME: keep a linked-list of empty entries, so we can use them again
   //        later. Right now null entries are left in ctx, env, name arys

   // first, make sure we actually have the entry
   rav_hash_elt_t *e;
   int e_is_target;
   rav_index_t idx = rav_lookup_(h, s, LOOKUP_FEATURE, &e, &e_is_target);
   if (idx==-1)
      return idx;

   // sanity check e/e_is_target
   rav_index_t e_idx;
   if (e_is_target) {
      e_idx = (rav_index_t)((e->ctx-h->ctx_ary)/h->dim);
      assert(e_idx==e);
   } else {
      e_idx = (rav_index_t)((e->next->ctx-h->ctx_ary)/h->dim);
      assert(e_idx==e);
   }

   // delete hash entry
   if (e_is_target) {
      // just invalidate hash entry (its not part of a linked-list)
      e->next = 0;
      e->ctx = 0;
   } else {
      // remove element from linked-list and free it
      rav_hash_elt_t *tgt = e->next;
      e->next = e->next->next;
      tgt->next = 0;
      tgt->ctx = 0;
      free(tgt);
   }

   // clear name, env, ctx arys
   double *d;
   int i;
   for (d = &h->env_ary[idx*h->dim], i=0; i<h->dim; i++)
      *d++ = 0.;
   for (d = &h->ctx_ary[idx*h->dim], i=0; i<h->dim; i++)
      *d++ = 0.;
   char *p = h->name_ary[idx];
   h->name_ary[idx] = 0;
   free(p);
   int s_len = strlen(p)+1;
   h->total_name_size -= s_len;

   return 0;
}

/* accumulating/binding */

void
rav_set(rav_hash_t *h, rav_index_t dst, rav_index_t src)
{
   double *d, *s;
   int i;
   for (d = &h->ctx_ary[dst*h->dim],
        s = &h->ctx_ary[src*h->dim],
        i=0; i<h->dim; i++)
      *d++ = *s++;
   for (d = &h->env_ary[dst*h->dim],
        s = &h->env_ary[src*h->dim],
        i=0; i<h->dim; i++)
      *d++ = *s++;
}

void
rav_add(rav_hash_t *h, rav_index_t dst, rav_index_t src)
{
   double *d, *s;
   int i;
   for (d = &h->ctx_ary[dst*h->dim],
        s = &h->env_ary[src*h->dim],
        i=0; i<h->dim; i++)
      *d++ += *s++;
}

void
rav_normalize(rav_hash_t *h, rav_index_t tgt)
{
   double *p;
   double *cur;
   double len;
   int i,j;

   if (tgt==-1) { // do them all
      for (cur=h->ctx_ary,
           i=0; i<h->cur_idx; i++) {
         len = 0.;
         for (p=cur,
              j=0; j<h->dim; j++, cur++)
            len += (*cur) * (*cur);
         len = sqrt(len);
         for (cur=p,
              j=0; j<h->dim; j++, cur++)
            if (*cur != 0.)
               *cur /= len;
      }
   } else {
      len = 0.;
      for (cur=&h->ctx_ary[tgt*h->dim],
           j=0; j<h->dim; j++, cur++)
         len += (*cur) * (*cur);
      len = sqrt(len);
      for (cur=&h->ctx_ary[tgt*h->dim],
           j=0; j<h->dim; j++, cur++)
         if (*cur != 0.)
            *cur /= len;
   }
}

void
rav_clr_ctx(rav_hash_t *h, rav_index_t tgt)
{
   double *p;
   int i;
   for (p=&h->ctx_ary[tgt*h->dim],
        i=0; i<h->dim; i++)
      *p++ = 0.;
}

void
rav_acc(rav_hash_t *h, rav_index_t src)
{
   double *d, *s;
   int i;
   for (d = &h->ctx_ary[h->cur_idx*h->dim],
        s = &h->env_ary[src*h->dim],
        i=0; i<h->dim; i++)
      *d++ += *s++;
}

void
rav_acc_from_ctx(rav_hash_t *h, rav_index_t src)
{
   double *d, *s;
   int i;
   for (d = &h->ctx_ary[h->cur_idx*h->dim],
        s = &h->ctx_ary[src*h->dim],
        i=0; i<h->dim; i++)
      *d++ += *s++;
}

void
rav_add_acc(rav_hash_t *h, rav_index_t dst)
{
   /* normalize accumulator first */
   rav_normalize(h, h->cur_idx);
   /* now add */
   int i;
   double *d, *p;
   for (p=&h->ctx_ary[h->cur_idx*h->dim],
        d=&h->ctx_ary[dst*h->dim],
        i=0; i<h->dim; i++)
      *d++ += *p++;
}

/* warning: not thread-safe */
#ifdef USE_FFT
static int did_fft_init=0;
fftw_plan pln_r2c_a;
fftw_plan pln_r2c_b;
fftw_plan pln_c2r_c;
double *inout_ac, *in_b;
fftw_complex *cmp_ac, *cmp_b;
#define NENTRIES (h->dim)
#define NENTRIES_CMP (h->dim+2)

void
rav_cconv_add(rav_hash_t *h, rav_index_t dst, rav_index_t x, rav_index_t y)
{
   /* circularly convolve a*b (by transforming into freq domain) and add to dst */
   double *c, *a, *b;
   int i;
   if (dst==-1) /* code for accumulator */
      dst = h->cur_idx;
   a=&h->env_ary[x*h->dim];
   b=&h->env_ary[y*h->dim];
   c=&h->ctx_ary[dst*h->dim];

   if (!did_fft_init) {
      inout_ac = (double*) fftw_malloc(sizeof(double)*NENTRIES);
      in_b = (double*) fftw_malloc(sizeof(double)*NENTRIES);
      cmp_ac = (fftw_complex*) fftw_malloc(sizeof(fftw_complex)*NENTRIES_CMP);
      cmp_b = (fftw_complex*) fftw_malloc(sizeof(fftw_complex)*NENTRIES_CMP);
      assert(inout_ac && in_b && cmp_ac && cmp_b);
      pln_r2c_a = fftw_plan_dft_r2c_1d(NENTRIES, inout_ac, cmp_ac, FFTW_ESTIMATE);
      pln_r2c_b = fftw_plan_dft_r2c_1d(NENTRIES, in_b, cmp_b, FFTW_ESTIMATE);
      pln_c2r_c = fftw_plan_dft_c2r_1d(NENTRIES, cmp_ac, inout_ac, FFTW_ESTIMATE);
      did_fft_init = 1;
   }
   /* copy in vectors */
   memcpy(inout_ac, a, sizeof(double)*NENTRIES);
   memcpy(in_b, b, sizeof(double)*NENTRIES);
   /* fft, mult in freq domain, inv-fft */
   fftw_execute(pln_r2c_a);
   fftw_execute(pln_r2c_b);
   for (i=0; i<NENTRIES; i++) {
      cmp_ac[i] = cmp_ac[i] * cmp_b[i];
   }
   fftw_execute(pln_c2r_c);
   /* scale back down and copy to dest */
   for (i=0; i<NENTRIES; i++) {
      c[i] += inout_ac[i] / NENTRIES;
   }
}
#else
void
rav_cconv_add(rav_hash_t *h, rav_index_t dst, rav_index_t a, rav_index_t b)
{
   /* circularly convolve a*b and add to dst */
   double *d, *x, *y;
   double acc;
   int i,j;
   if (dst==-1) /* code for accumulator */
      dst = h->cur_idx;
   for (x=&h->env_ary[a*h->dim],
        y=&h->env_ary[b*h->dim],
        d=&h->ctx_ary[dst*h->dim],
        i=0; i<h->dim; i++) {
      acc = 0.;
      for (j=0; j<h->dim; j++) {
         //acc += x[j] * y[(i-j)&RAV_ELT_VECT_MASK];
         if ((i-j)>=0)
            acc += x[j] * y[i-j];
         else
            acc += x[j] * y[h->dim+(i-j)];
      }
      *d++ += acc;
   }
}
#endif

/* misc helpers for collecting nearest results */

typedef struct sort_info {
   rav_index_t idx;
   double score;
   char *name;
} sort_info_t;

/* warning: not thread-safe or even multiple-object-safe, unless results are
 * retreived immediately after calling rav_closest.
 */
int lstn = 0;
sort_info_t* wlst = 0;
sort_info_t* flst = 0;
sort_info_t* vlst = 0;

void
alloc_sort_info(sort_info_t** lstp, int n)
{
   if (n != lstn) {
      if (*lstp)
         free(*lstp);
      *lstp = (sort_info_t*)calloc(n, sizeof(sort_info_t));
      if (*lstp==0) {
         fprintf(stderr, "alloc_sort_info: calloc failed\n");
         return;
      }
   }

   int i;
   for (i=0; i<n; i++) {
      (*lstp)[i].score = -1e10;
      (*lstp)[i].name = "-";
      (*lstp)[i].idx = -1;
   }
}

void
sorted_insert(sort_info_t *lst, int n, double cosine, char *name, rav_index_t idx)
{
   /* For large n, we can do better than linear search through a list - 
    * even now once the list is full we would do better going max-to-min
    * so we can discard faster. However, with n=10 even if we completely
    * remove this function it only saves 2% */
   int i,j;
   for (i=0; i<n; i++) {
      if (cosine > lst[i].score) {
         for (j=n-1; j>i; j--)
            lst[j] = lst[j-1];
         lst[i].score = cosine;
         lst[i].name = name;
         lst[i].idx = idx;
         break;
      }
   }
}

/* searching */

double
rav_cosine(rav_hash_t *h, rav_index_t idx1, rav_index_t idx2)
{
   int i;
   double *ctx1 = &h->ctx_ary[idx1*h->dim];
   double *ctx2 = &h->ctx_ary[idx2*h->dim];
   double totacc = 0;

   for (i=0; i<h->dim; i++) {
      totacc += (*ctx1) * (*ctx2);
      ctx1++;
      ctx2++;
   }
   return totacc;
}

void
rav_closest(rav_hash_t *h, rav_index_t tgt_idx, int n, int include_tgt)
{
   if (tgt_idx<0 || n<=0) {
      fprintf(stderr, "rav_closest: illegal params\n");
      return;
   }

   alloc_sort_info(&wlst, n);
   alloc_sort_info(&flst, n);
   alloc_sort_info(&vlst, n);
   if (wlst==0 || flst==0 || vlst==0) {
      if (wlst != 0) {
         free(wlst);
         wlst = 0;
      }
      if (flst != 0) {
         free(flst);
         flst = 0;
      }
      if (vlst != 0) {
         free(vlst);
         vlst = 0;
      }
      return;
   }
   lstn = n;

   int i,j;

   double *tgt_s;
   double *tgt, *cur;
   double acc, cosine;

   for (tgt_s = &h->ctx_ary[tgt_idx*h->dim],
        cur = h->ctx_ary,
        i=0; i<h->cur_idx; i++) {
      if (include_tgt || (tgt_s!=cur)) {
         if (get_freq_code(h, i)!=FREQ_OK)
            continue;
         acc = 0.;
         for (tgt = tgt_s,
              j=0; j<h->dim; j++)
            acc += (*tgt++) * (*cur++);
         cosine = acc;
         /* insert into sorted list. $=feature, #=value, else word */
         char *nm = h->name_ary[i];
         if (nm) {
            if (nm[0] == '$')
               sorted_insert(flst, n, cosine, nm, i);
            else if (nm[0] == '#')
               sorted_insert(vlst, n, cosine, nm, i);
            else
               sorted_insert(wlst, n, cosine, nm, i);
         }
      }
   }
}

double
rav_closest_score_n(rav_hash_t *h, int type, int n)
{
   switch (type) {
   case RAV_WORD:
      if (wlst && n<lstn)
         return wlst[n].score;
      break;
   case RAV_FEATURE:
      if (flst && n<lstn)
         return flst[n].score;
      break;
   case RAV_VALUE:
      if (vlst && n<lstn)
         return vlst[n].score;
      break;
   }
   return 0.;
}

rav_index_t
rav_closest_idx_n(rav_hash_t *h, int type, int n)
{
   switch (type) {
   case RAV_WORD:
      if (wlst && n<lstn)
         return wlst[n].idx;
      break;
   case RAV_FEATURE:
      if (flst && n<lstn)
         return flst[n].idx;
      break;
   case RAV_VALUE:
      if (vlst && n<lstn)
         return vlst[n].idx;
      break;
   }
   return -1;
}

char *
rav_closest_name_n(rav_hash_t *h, int type, int n)
{
   switch (type) {
   case RAV_WORD:
      if (wlst && n<lstn)
         return wlst[n].name;
      break;
   case RAV_FEATURE:
      if (flst && n<lstn)
         return flst[n].name;
      break;
   case RAV_VALUE:
      if (vlst && n<lstn)
         return vlst[n].name;
      break;
   }
   return 0;
}

/* saving/loading/debug */

int
rav_save(rav_hash_t *h, int fileno)
{
   unsigned long n = sizeof(int);
   int tmp = sizeof(rav_hash_t);
   if (write(fileno, &tmp, n) != n) {
      perror("error saving rav_hash size");
      return -1;
   }

   /* save global hash info structure */
   n = sizeof(rav_hash_t);
   if (write(fileno, h, n) != n) {
      perror("error saving h");
      return -1;
   }

   /* save big memories */
   n = h->cur_idx*sizeof(double)*h->dim;
   unsigned long n_saved = 0;
   //if (write(fileno, h->env_ary, n) != n) {
   if ((n_saved = write(fileno, h->env_ary, n)) != n) {
      perror("error saving env_ary");
      fprintf(stderr, "tried to save %ld, but only saved %ld\n", n, n_saved);
      return -1;
   }
   if (write(fileno, h->ctx_ary, n) != n) {
      perror("error saving ctx_ary");
      return -1;
   }

   /* save freq code memory */
   n = h->low_freq_size*sizeof(uint64_t);
   if (write(fileno, h->low_freq, n) != n) {
      perror("error saving low_freq");
      return -1;
   }

   /* serialize hash elements */
   int i;
   uintptr_t p;
   rav_hash_elt_t *e;
   for (e = h->hash_ary,
        i=0; i<RAV_HASH_ENTRIES; i++, e++) {
      if (e->ctx!=0) {
         rav_hash_elt_t *chain;
         for (chain=e; chain!=0; chain=chain->next) {
            n = sizeof(int);
            if (write(fileno, &i, n) != n) {
               perror("error saving hash_ary index");
               return -1;
            }
            n = sizeof(uintptr_t);
            p = (chain->ctx-h->ctx_ary);
            if (write(fileno, &p, n) != n) {
               perror("error saving hash_elt ctx offset");
               return -1;
            }
         }
      }
   }
   n = sizeof(int);
   tmp = RAV_HASH_ENTRIES;
   if (write(fileno, &tmp, n) != n) {
      perror("error saving final hash_ary index");
      return -1;
   }

   /* serialize names */
   char **c;
   int total_len = 0;
   for (c = h->name_ary,
        i=0; i<h->cur_idx; i++, c++) {
      if (*c!=0) {
         size_t l = strlen(*c)+1;
         total_len += l;
         n = sizeof(size_t);
         if (write(fileno, &l, n) != n) {
            perror("error saving name length");
            return -1;
         }
         n = (unsigned long)l;
         if (write(fileno, *c, n) != n) {
            perror("error saving name");
            return -1;
         }
      }
   }
   assert(total_len==h->total_name_size);
   return 0;
}

int
rav_load(rav_hash_t *h, int fileno)
{
   /* first read/verify global info struct */
   unsigned long n = sizeof(int);
   ssize_t nread;
   int size;
   if ((nread = read(fileno, &size, n)) != n) {
      perror("error loading rav_hash size");
      return -1;
   }
   n = sizeof(rav_hash_t);
   if (size != n) {
      perror("rav_hash_t size does not match compiled version");
      return -1;
   }

   rav_hash_t tmph;
   if ((nread = read(fileno, &tmph, n)) != n) {
      perror("error loading rav_hash");
      return -1;
   }
   if (tmph.max_elts > h->max_elts) {
      fprintf(stderr, "error loading rav_hash - max_elts too high (%d %d)", 
            tmph.max_elts, h->max_elts);
      return -1;
   }
   if (tmph.dim != h->dim) {
      fprintf(stderr, "error loading rav_hash - dimensions do not agree (%d %d)",
            tmph.dim, h->dim);
      return -1;
   }
   h->cur_idx = tmph.cur_idx;
   h->total_name_size = tmph.total_name_size;
   h->orig_random_seed = tmph.orig_random_seed;
   h->cur_random_seed = tmph.cur_random_seed;

   /* read big memories */
   n = h->cur_idx*sizeof(double)*h->dim;
   if ((nread = read(fileno, h->env_ary, n)) != n) {
      perror("error loading env_ary");
      return -1;
   }
   if ((nread = read(fileno, h->ctx_ary, n)) != n) {
      perror("error loading ctx_ary");
      return -1;
   }

   /* read low_freq memory */
   n = h->low_freq_size*sizeof(uint64_t);
   if ((nread = read(fileno, h->low_freq, n)) != n) {
      perror("error loading low_freq");
      return -1;
   }

   /* free current hash elements */
   rav_free_elts(h);

   /* deserialize hash elements */
   bzero(h->hash_ary, RAV_HASH_ENTRIES*sizeof(rav_hash_elt_t));

   int prev_i;
   int i;
   rav_hash_elt_t *e;

   for (i=0, prev_i=-1, e=0;;) {
      n = sizeof(int);
      /* get next index */
      if ((nread = read(fileno, &i, n)) != n) {
         perror("error loading hash_ary index");
         return -1;
      }
      if (i>=RAV_HASH_ENTRIES)
         break;
      if (prev_i==i) {
         assert(e);
         e->next = (rav_hash_elt_t*)malloc(sizeof(rav_hash_elt_t));
         if (e->next==0) {
            perror("error hash elt malloc failed during load");
            return -1;
         }
         e = e->next;
         e->next = 0;
         e->ctx = 0;
      } else {
         e = &h->hash_ary[i];
         e->next = 0;
      }
      /* read index */
      uintptr_t p;
      n = sizeof(uintptr_t);
      if ((nread = read(fileno, &p, n)) != n) {
         perror("error loading hash_ary ctx offset");
         return -1;
      }
      e->ctx = h->ctx_ary+p;
      prev_i = i;
   }

   /* free current names */
   rav_free_names(h);

   /* deserialize names */
   size_t slen;
   int total_len = 0;
   for (i=0; i<h->cur_idx; i++) {
      n = sizeof(size_t);
      if ((nread = read(fileno, &slen, n)) != n) {
         perror("error loading name length");
         return -1;
      }
      total_len += slen;
      h->name_ary[i] = (char*)malloc(slen);
      if (h->name_ary[i]==0) {
         perror("error name malloc failed during load");
         return -1;
      }
      n = slen;
      if ((nread = read(fileno, h->name_ary[i], n)) != n) {
         perror("error loading name");
         return -1;
      }
   }
   if (total_len != h->total_name_size) {
      perror("error in total name length");
      return -1;
   }
   return 0;
}

void 
rav_dbg(rav_hash_t *h, rav_index_t idx)
{
   if (idx==-1)
      idx=h->cur_idx;
   if (idx > h->cur_idx)
      return;

   double *p;
   int i;
   if (idx==h->cur_idx)
      printf("accumulate: ");
   else
      printf("%10s: ", h->name_ary[idx]);
   for (p = &h->env_ary[idx*h->dim], i=0; i<4; i++)
      printf("%+6.4f ", p[i]);
   fputs("- ", stdout);
   for (p = &h->ctx_ary[idx*h->dim], i=0; i<4; i++)
      printf("%+6.4f ", p[i]);
   fputs("\n", stdout);
}

void 
rav_dump(rav_hash_t *h, const char *fname, int n, int include_values)
{
   int i;
   if (n<=0 || n>h->cur_idx)
      n=h->cur_idx;
   /* open file and write header */
   FILE *f = fopen(fname, "wb+");
   if (f==0)
      return;
   /* dump up to n ravs */
   for (i=0; i<n; i++) {
      if (get_freq_code(h, i)==FREQ_OK && 
            (include_values || h->name_ary[i][0]!='#'))
         fwrite(&h->ctx_ary[i*h->dim], sizeof(double), h->dim, f);
   }
   fclose(f);
}

void 
rav_dump_names(rav_hash_t *h, const char *fname, int n, int include_values)
{
   int i;
   if (n<=0 || n>h->cur_idx)
      n=h->cur_idx;
   /* dump names */
   FILE *f = fopen(fname, "w+");
   if (f==0)
      return;
   fprintf(f, "# -*- coding: iso-8859-15 -*-\n");
   fprintf(f, "dim = %d\n", h->dim);
   fprintf(f, "names = [\\\n");
   for (i=0; i<n; i++) {
      if (get_freq_code(h, i)==FREQ_OK &&
            (include_values || h->name_ary[i][0]!='#'))
         fprintf(f, "  \"%s\",\n", h->name_ary[i]);
   }
   fprintf(f, "]\n");
   fclose(f);
}

/* end of rav.c */
